# -*- coding: utf-8 -*-

'''
Created on 26/11/2011

@author: Luis M. Ugalde
www.forondarena.net

Copyright © 2011 Luis M. Ugalde
Licensed under the terms of the MIT License
(see LICENSE file for details)

This software can't be considered to be even in Alpha state. It still needs a lot of work.
'''


from . import abstractparser

import re
import zmq
import pickle
import uuid
from parsers.parserexception import ParserException


class ParserPostfix(abstractparser.AbstractParser):
    '''
    Class to parse Postfix log files.
    This parser does not handle antispam/antivirus or any other add-on log line.
    This parser should be extended with a per-site regular expression group.
    '''
    
    
    def __init__(self, helperfilename = None):
        '''
        Define the available regular expressions for the parser.
        The helper filename is used to record unparseable lines. This will allow the sysadmin to review
        and improve the ruleset.
        '''
        
        print self.__class__.__name__ + ": ParserPostfix instance created"

        _beginregexp = '^(?P<date>\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<mailserver>.+)\s+postfix'
        _noqueueregexp = '/smtpd\[\d+\]:\s+NOQUEUE:\s+reject:\s+RCPT\s+from\s+(?P<client>\S+):\s+'

        self.nologconnfrom =  re.compile(_beginregexp + '/smtpd\[\d+\]: connect from.*\]$')
        self.nologtlsfrom =   re.compile(_beginregexp + '/smtpd\[\d+\]: setting up TLS connection from.*\]$')
        self.nologdisconn =   re.compile(_beginregexp + '/smtpd\[\d+\]: disconnect from.*\]$')
        self.nologanonymous = re.compile(_beginregexp + '/smtpd\[\d+\]: Anonymous TLS connection established from.*bits\)$') 
        self.nologlost =      re.compile(_beginregexp + '/smtpd\[\d+\]: lost connection after.*from.*\]$')
        self.nologwarning =   re.compile(_beginregexp + '/(smtpd|cleanup|smtp)\[\d+\]: warning:')
        self.nologmanyerrors = re.compile(_beginregexp + '/smtpd\[\d+\]: too many errors after.*\]$')
        self.nologtimeout =   re.compile(_beginregexp + '/smtpd\[\d+\]: timeout after.*\]$')

        self.noqueuerblregexp =           re.compile(_beginregexp + _noqueueregexp + '554\s+5\.7\.1\s+Service\s+unavailable;\s+Client\s+host\s+\[(?P<srcip>\S+)\].+\s+from=<(?P<from>\S*)>\s+to=<(?P<to>\S+)>\s+proto=(?P<proto>\S+)\s+helo=<(?P<helo>\S+)>$')
        self.noqueueunknownuserregexp =   re.compile(_beginregexp + _noqueueregexp + '550\s+5\.1\.1.+Recipient\s+address\s+rejected:\s+User\s+unknown\s+in\s+\S+\s+recipient\s+table;\s+from=<(?P<from>\S*)>\s+to=<(?P<to>\S+)>\s+proto=(?P<proto>\S+)\s+helo=<(?P<helo>\S+)>$')
        self.noqueueunknowndomainregexp = re.compile(_beginregexp + _noqueueregexp + '450\s+4\.1\.8.+Sender\s+address\s+rejected:\s+Domain\s+not\s+found;\s+from=<(?P<from>\S*)>\s+to=<(?P<to>\S+)>\s+proto=(?P<proto>\S+)\s+helo=<(?P<helo>\S+)>$')
        self.noqueueheloregexp =          re.compile(_beginregexp + _noqueueregexp + '554\s+5\.7\.1.+Helo\s+command\s+rejected:.*\s+from=<(?P<from>\S*)>\s+to=<(?P<to>\S+)>\s+proto=(?P<proto>\S+)\s+helo=<(?P<helo>\S+)>$')
        self.noqueuerelayregexp =         re.compile(_beginregexp + _noqueueregexp + '554\s+5\.7\.1.+Relay\s+access\s+denied;\s+from=<(?P<from>\S*)>\s+to=<(?P<to>\S+)>\s+proto=(?P<proto>\S+)\s+helo=<(?P<helo>\S+)>$')
        
        self.newconnregexp = re.compile(_beginregexp + '/smtpd\[\d+\]:\s+(?P<queueid>\w+):\s+client=(?P<client>\S+)$')
        self.msgregexp =     re.compile(_beginregexp + '/cleanup\[\d+\]:\s+(?P<queueid>\w+):\s+message-id=(?P<msgid>\S*)$')
        self.fromregexp =    re.compile(_beginregexp + '/qmgr\[\d+\]:\s+(?P<queueid>\w+):\s+from=<(?P<from>\S*)>,\s+size=(?P<size>\d+),\s+nrcpt=(?P<nrcpt>\d+)\s+\(queue\s+active\)$')

        self.smtpregexp =    re.compile(_beginregexp + '/smtp\[\d+\]:\s+(?P<queueid>\w+):\s+to=<(?P<to>\S+)>,(\s+orig_to=<(?P<orig_to>\S+)>,)?\s+relay=(?P<relay>\S+),(\s+conn_use=\d+,)?\s+delay=(?P<delay>\S+),.+,\s+status=(?P<status>.+)$')
        self.localregexp =   re.compile(_beginregexp + '/local\[\d+\]:\s+(?P<queueid>\w+):\s+to=<(?P<to>\S+)>,(\s+orig_to=<(?P<orig_to>\S+)>,)?\s+relay=(?P<relay>\S+),(\s+conn_use=\d+,)?\s+delay=(?P<delay>\S+),\s+status=(?P<status>.+)$')
        self.piperegexp =    re.compile(_beginregexp + '/pipe\[\d+\]:\s+(?P<queueid>\w+):\s+to=<(?P<to>\S+)>,(\s+orig_to=<(?P<orig_to>\S+)>,)?\s+relay=(?P<relay>\S+),(\s+conn_use=\d+,)?\s+delay=(?P<delay>\S+),.+,\s+status=(?P<status>.+)$')

        self.endregexp = re.compile(_beginregexp + '/qmgr\[\d+\]:\s+(?P<queueid>\w+):\s+removed$')
        
        abstractparser.AbstractParser.__init__(self)
        
        self.setHelper(helperfilename)


    def consumeLine(self, line):
        usehelper = self.useHelper()
        
        if line == 'NODATA':
            tmpbuffer = ['NODATA', 'NODATA', '', {}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            print "{}-{}: Sent NODATA signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchnologconnfrom = self.nologconnfrom.match(line)
        if matchnologconnfrom: return
            
        matchnologtlsfrom = self.nologtlsfrom.match(line)
        if matchnologtlsfrom: return
            
        matchnologdisconn = self.nologdisconn.match(line)
        if matchnologdisconn: return
            
        matchnologanonymous = self.nologanonymous.match(line)
        if matchnologanonymous: return
            
        matchnologlost = self.nologlost.match(line)
        if matchnologlost: return

        matchnologwarning = self.nologwarning.match(line)
        if matchnologwarning: return

        matchmanyerrors = self.nologmanyerrors.match(line)
        if matchmanyerrors: return

        matchnologtimeout = self.nologtimeout.match(line)
        if matchnologtimeout: return

        matchnoqueuerbl = self.noqueuerblregexp.match(line)
        if matchnoqueuerbl:
            linebits = matchnoqueuerbl.groupdict()
            tmpbuffer = ['SINGLELINE', 'RBL', linebits['mailserver'] + '_' + str(uuid.uuid1()), 
                         {'datein': linebits['date'], 'client': linebits['client'], 'rcptto': linebits['to'], 
                          'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'status': 'Rejected RBL: ' + linebits['srcip'], 'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent SINGLE RBL signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchunknownuser = self.noqueueunknownuserregexp.match(line)
        if matchunknownuser:
            linebits = matchunknownuser.groupdict()
            tmpbuffer = ['SINGLELINE', 'USER', linebits['mailserver'] + '_' + str(uuid.uuid1()), 
                         {'datein': linebits['date'], 'client': linebits['client'], 'rcptto': linebits['to'], 
                          'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'status': 'Rejected User Unknown: ' + linebits['to'], 'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent SINGLE USER signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchunknowndomain = self.noqueueunknowndomainregexp.match(line)
        if matchunknowndomain:
            linebits = matchunknowndomain.groupdict()
            tmpbuffer = ['SINGLELINE', 'DOMAIN', linebits['mailserver'] + '_' + str(uuid.uuid1()), 
                         {'datein': linebits['date'], 'client': linebits['client'], 'rcptto': linebits['to'], 
                          'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'status': 'Rejected Unknown Domain: ' + linebits['to'], 'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent SINGLE DOMAIN signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchhelorejected = self.noqueueheloregexp.match(line)
        if matchhelorejected:
            linebits = matchhelorejected.groupdict()
            tmpbuffer = ['SINGLELINE', 'HELO', linebits['mailserver'] + '_' + str(uuid.uuid1()), 
                         {'datein': linebits['date'], 'client': linebits['client'], 'rcptto': linebits['to'], 
                          'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'status': 'Rejected Helo: ' + linebits['helo'], 'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent SINGLE HELO signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchrelaydenied = self.noqueuerelayregexp.match(line)
        if matchrelaydenied:
            linebits = matchrelaydenied.groupdict()
            tmpbuffer = ['SINGLELINE', 'RELAY', linebits['mailserver'] + '_' + str(uuid.uuid1()), 
                         {'datein': linebits['date'], 'client': linebits['client'], 'rcptto': linebits['to'], 
                          'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'status': 'Rejected Relay denied: from ' + linebits['from'] if linebits['from'] else "MAILER-DAEMON" + ' to '+ linebits['to'], 
                          'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent SINGLE RELAY signal.".format(self.__class__.__name__, self.parserbindaddress)
            return

        matchnewconn = self.newconnregexp.match(line)
        if matchnewconn:
            linebits = matchnewconn.groupdict()
            tmpbuffer = ['KEYDATA', 'NEWID', linebits['mailserver'] + '_' + linebits['queueid'], 
                         {'datein': linebits['date'], 'client': linebits['client']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEY DATA, NEWID signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchmsg = self.msgregexp.match(line)
        if matchmsg:
            linebits = matchmsg.groupdict()
            tmpbuffer = ['KEYDATA', 'DATA', linebits['mailserver'] + '_' + linebits['queueid'], 
                         {'msgid': linebits['msgid'] if linebits['msgid'] else ""}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEY DATA, msg signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchfrom = self.fromregexp.match(line)
        if matchfrom:
            linebits = matchfrom.groupdict()
            tmpbuffer = ['KEYDATA', 'DATA', linebits['mailserver'] + '_' + linebits['queueid'], 
                         {'mailfrom': linebits['from'] if linebits['from'] else "MAILER-DAEMON", 
                          'nrcpt': linebits['nrcpt'], 'size': linebits['size']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEY DATA, mail from signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchsmtp = self.smtpregexp.match(line)
        if matchsmtp:
            linebits = matchsmtp.groupdict()
            tmpbuffer = ['KEYDATAMULTIPLE', 'rcptto', linebits['mailserver'] + '_' + linebits['queueid'], 
                         {'dateout': linebits['date'], 'rcptto': linebits['to'], 
                          'relay': linebits['relay'], 'status': linebits['status'], 'delay': linebits['delay'], 'origto': linebits['orig_to']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEYDATAMULTIPLE RCPTTO signal.".format(self.__class__.__name__, self.parserbindaddress)
            return

        matchlocal = self.localregexp.match(line)
        if matchlocal:
            linebits = matchlocal.groupdict()
            tmpbuffer = ['KEYDATAMULTIPLE', 'rcptto', linebits['mailserver'] + '_' + linebits['queueid'],
                         {'dateout': linebits['date'], 'rcptto': linebits['to'], 'relay': linebits['relay'], 
                          'status': linebits['status'], 'delay': linebits['delay'], 'origto': linebits['orig_to']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEYDATAMULTIPLE RCPTTO signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchpipe = self.piperegexp.match(line)
        if matchpipe:
            linebits = matchpipe.groupdict()
            tmpbuffer = ['KEYDATAMULTIPLE', 'rcptto', linebits['mailserver'] + '_' + linebits['queueid'], 
                         {'dateout': linebits['date'], 'rcptto': linebits['to'], 'relay': linebits['relay'], 
                          'status': linebits['status'], 'delay': linebits['delay'], 'origto': linebits['orig_to']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEYDATAMULTIPLE RCPTTO signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        matchend = self.endregexp.match(line)
        if matchend:
            linebits = matchend.groupdict()
            tmpbuffer = ['KEYDATA', 'ENDID', linebits['mailserver'] + '_' + linebits['queueid'],
                         {'dateend': linebits['date']}]
            pickledbuffer = pickle.dumps(tmpbuffer)
            self.parserbindsocket.send(pickledbuffer)
            if __debug__:
                print "{}-{}: Sent KEYDATA Message END signal.".format(self.__class__.__name__, self.parserbindaddress)
            return
        
        if usehelper:
            self.writeHelper(line + "\n")
            if __debug__:
                print "{}-{}: Unparseable line. Writing to helper file: {}".format(self.__class__.__name__, self.parserbindaddress, self.filename)


    def parseLines(self):
        '''
        Process a line previously read from a reader.
        Create a message with the parsed tokens, and send it to the state store.
        '''
        
        mustexit = 0
        idlesockets = 0

        while True:
            socks = dict(self.poller.poll(3000))
            if socks:
                idlesockets = 0
                for i in self.readerconnectsockets:
                    if i in socks and socks.get(i) == zmq.POLLIN:
                        line = i.recv()
                        if __debug__:
                            print "{}-{}: The line is {}.".format(self.__class__.__name__, self.parserbindaddress, line)
                        self.consumeLine(line)
                if self.readerconnectnotifysocket in socks and socks.get(self.readerconnectnotifysocket) == zmq.POLLIN:
                    line = self.readerconnectnotifysocket.recv()
                    if __debug__:
                        print "{}-{}: The line is {}.".format(self.__class__.__name__, self.parserbindnotifyaddress, line)
                    if line == 'END':
                        print self.__class__.__name__ + ": Received END signal. Setting END flag."
                        mustexit = 1
            else:
                if mustexit > 0:
                    print "{}-{}: No data left at parser for 3 seconds with end signal. Ending.".format(self.__class__.__name__, self.parserbindaddress)
                    self.parserbindnotifysocket.send('END')
                    print "{}-{}: Sent END signal to {}.".format(self.__class__.__name__, self.parserbindaddress, self.parserbindnotifyaddress)
                    break
                if idlesockets > 3:
                    print "{}-{}: Too long with no activity from sockets. Should probably do something. TODO.".format(self.__class__.__name__, self.parserbindaddress)
                    idlesockets = 0
                idlesockets += 1

        self.closeSources()
        self.closeNotifySources()
        self.closeBindParser()
        self.closeBindNotifyParser()
        self.closeContext()
        self.closeHelper()


if __name__ == '__main__':
    print "Please, do not run this"
